import os
import re
import urllib2
from BeautifulSoup import BeautifulSoup as BS
from BeautifulSoup import BeautifulStoneSoup as BSS
from ConfigParser import ConfigParser

config = ConfigParser()
config.read([os.path.expanduser('earlbot.cfg')])

# URILEN is used to set a lenght of url that will be passed to tinyurl
URILEN = config.get('bot','URILEN')

def main(self, sender, channel, flags, message, DB, DEBUG):

    #right now only http/https/ftp are being parsed
    uri_prefix = '(http://|https://|ftp://)(\S*)'
    uri_re = re.compile(uri_prefix)

    for uri in uri_re.findall(message):

        uri = ''.join(uri)
	request = urllib2.Request(uri)

	# add a user agent to the request, allows grabbing google results etc.
	request.add_header('User-Agent','Mozilla/5.0 \(X11; U; Linux x86_64; en-US; rv:1.9.0.4\) Gecko/2008112309 Iceweasel/3.0.4 \(Debian-3.0.4-1\)')

	#actually retrieve the url contents
        site_contents = urllib2.urlopen(request)

	#pass contents of url (raw html) to beautiful soup to extract <title>
        soup = BS(site_contents)

        try:

	    #grab the text foo in <title>foo</title> tags
            title_html = soup.html.head.title.string

	    #convert stuff like &amp; to unicode
            title_unicode = BSS(title_html, convertEntities="html").contents[0]

	    #strip out spaces, tabs, newlines etc., (thanks matt!)
	    title_unicode = re.sub('\s+',' ',title_unicode)

	    #check URILEN, if url is too long, send to tinyurl
            if len(uri) > int(URILEN):
                tinyurl = urllib2.urlopen('http://tinyurl.com/api-create.php?url=%s' %(uri)).readlines()
                message = title_unicode + ': %s' %(tinyurl[0])

            else:
                message = title_unicode
        
		#for debugging, set with DEBUG in earlbot.cfg
                 if DEBUG == True:
                    print title_unicode
                    print message

	    #send the title to the channel
            self.send_channel_message(channel, "Title: %s" %(message))

        except:
	    #TODO: perhaps implement a channel message detailing errors
            print "no title for that document"

        
        
